import requests
from lxml import etree
from lxml import html
import pandas as pd
import time
import json
import os
from datetime import datetime
import log

# 配置日志
log.basicConfig(
    level=log.INFO,
    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
    handlers=[log.FileHandler("weather_scraper.log"), log.StreamHandler()]
)
logger = log.getLogger("./log/weather_scraper")

DATA_DIR = "weather_data"
os.makedirs(DATA_DIR, exist_ok=True)

def get_html(city, month):
    headers = {
        "Accept-Encoding": "Gzip",
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36",
        }
    url = f'https://lishi.tianqi.com/{city}/{month}.html'

    r = requests.get(url, headers=headers)
    # print(r)
    r_html = etree.HTML(r.text)

    return r_html



def save_weather_data(data, city, month):
    """保存天气数据到JSON文件"""
    if data is None or len(data) == 0:
        logger.warning(f"No data to save for {city} in {month}")
        return False

        # 确保城市目录存在
    city_dir = os.path.join(DATA_DIR, city)
    os.makedirs(city_dir, exist_ok=True)

    # 保存到JSON文件
    filename = os.path.join(city_dir, f"{month}.json")

    try:
        with open(filename, 'w', encoding='utf-8') as f:
            json.dump(data, f, ensure_ascii=False, indent=2)
        logger.info(f"Saved data to {filename}")
        return True
    except Exception as e:
        logger.error(f"Error saving data: {e}")
        return False



for city in ["beijing", "guangzhou", "shenzhen","shanghai"]:
        # 月份参数列表
    month_list = pd.period_range('202411','202411',freq='M').strftime('%Y%m')
    # print(month_list)

    data = []
    for i , month in enumerate(month_list):
        time.sleep(1)
        r_html = get_html(city, month)
        # 找到存放历史天气数据的div节点
        # print(html.tostring(r_html, pretty_print=True).decode())
        try:
            div = r_html.xpath('.//div[@class="tian_three"]')
            if div:
                div = div[0]
                # print out the div for further debugging
                # print(html.tostring(div, pretty_print=True).decode())
                lis = div.xpath('.//li')
                for li in lis:
                    item = {
                        'date':li.xpath('./div[@class="th200"]/text()')[0].split(' ')[0].strip() ,
                        'high_temp':li.xpath('./div[@class="th140"]/text()')[0],
                        'low_temp':li.xpath('./div[@class="th140"]/text()')[1],
                        'weather':li.xpath('./div[@class="th140"]/text()')[2],
                        'wind_direction':li.xpath('./div[@class="th140"]/text()')[3]
                        }
                    # df = df.append(item, ignore_index=True)
                    data.append(item)
                print(f'共{i+1}月数据已采集')
            else:
                print(f"No div with class 'tian_three' found for month {month}")
        except Exception as e:
            print(f"An error occurred while processing month {month}: {e}")

        save_weather_data(data, city, month)